#! /usr/bin/env python
# -*- coding: utf-8 -*-

import requests
import random
import time
import json
import pymysql

# http请求函数
# @param tweet_id 目标贴文的twid
# @pram user_id 用户自己的账号id
# @param cursor_value 下一页的游标
# @param status_code 请求返回码
def x_request(tweet_id, user_id, tw_cursor, db_cursor):

    status_code = 0
    cursor_value = ''

    url = 'https://x.com/i/api/graphql/INsneb6y78uXRviWsuA-Rw/TweetDetail?variables=%7B%22focalTweetId%22%3A%22'+tweet_id+'%22%2C%22cursor%22%3A%22'+tw_cursor+'%22%2C%22with_rux_injections%22%3Afalse%2C%22rankingMode%22%3A%22Relevance%22%2C%22includePromotedContent%22%3Atrue%2C%22withCommunity%22%3Atrue%2C%22withQuickPromoteEligibilityTweetFields%22%3Atrue%2C%22withBirdwatchNotes%22%3Atrue%2C%22withVoice%22%3Atrue%7D&features=%7B%22rweb_video_screen_enabled%22%3Afalse%2C%22profile_label_improvements_pcf_label_in_post_enabled%22%3Atrue%2C%22rweb_tipjar_consumption_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22premium_content_api_read_enabled%22%3Afalse%2C%22communities_web_enable_tweet_community_results_fetch%22%3Atrue%2C%22c9s_tweet_anatomy_moderator_badge_enabled%22%3Atrue%2C%22responsive_web_grok_analyze_button_fetch_trends_enabled%22%3Afalse%2C%22responsive_web_grok_analyze_post_followups_enabled%22%3Atrue%2C%22responsive_web_jetfuel_frame%22%3Afalse%2C%22responsive_web_grok_share_attachment_enabled%22%3Atrue%2C%22articles_preview_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Atrue%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22responsive_web_grok_show_grok_translated_post%22%3Afalse%2C%22responsive_web_grok_analysis_button_from_backend%22%3Afalse%2C%22creator_subscriptions_quote_tweet_preview_enabled%22%3Afalse%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_grok_image_annotation_enabled%22%3Atrue%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D&fieldToggles=%7B%22withArticleRichContentState%22%3Atrue%2C%22withArticlePlainText%22%3Afalse%2C%22withGrokAnalyze%22%3Afalse%2C%22withDisallowedReplyControls%22%3Afalse%7D'
    params = {
    }

    # 换成自己的浏览器headers
    headers = {
        'key': 'value'
    }

    # 换成自己的cookies
    cookies = {
        'key': 'value'
    }

    # 请求数据为空的情况下（GET请求）就可以直接发送请求
    response = requests.get(url, headers=headers, cookies=cookies, params=params)
    status_code = response.status_code
    # print(response.text)
    # 格式化为json格式
    try:
        result = response.json()
    except json.JSONDecodeError:
        # print('网络代码:'+str(response.status_code))
        return cursor_value, status_code
    
    # # 提取entries列表
    instructions = result.get("data", {}) \
                    .get("threaded_conversation_with_injections_v2", {}) \
                    .get("instructions", [])
    entries = None
    for item in instructions:
        if item.get("type") == "TimelineAddEntries":
            entries = item.get("entries", [])
            break
    # 遍历提取entries列表，提取贴文内容
    for entry in entries:
        content = entry.get("content", {})
        
        # 页面游标
        if content.get("__typename", "") == 'TimelineTimelineItem' and \
            content.get("itemContent", {}).get("__typename", "") == 'TimelineTimelineCursor' and \
            content.get("itemContent", {}).get("cursorType", "") == 'Bottom':
            cursor_value = content.get("itemContent", {}).get("value", "")
        # 贴文内容
        elif content.get("__typename", "") == 'TimelineTimelineModule':
            timeline_add_entries_content_items = content.get("items")
            if timeline_add_entries_content_items and isinstance(timeline_add_entries_content_items, list):
                for timeline_add_entries_content_item in timeline_add_entries_content_items:
                    timeline_add_entries_legacy = timeline_add_entries_content_item.get("item", {})\
                                                                                    .get("itemContent", {})\
                                                                                    .get("tweet_results", {})\
                                                                                    .get("result", {})\
                                                                                    .get("legacy", {})
                    # 只选非本人的消息
                    if timeline_add_entries_legacy.get("id_str", "") != "" and timeline_add_entries_legacy.get("user_id_str", "") != user_id:
                        # 筛选掉广告消息
                        if len(timeline_add_entries_legacy.get("entities", {}).get("user_mentions",[])) > 0:
                            sql = "insert into replys (src_twid, twid, full_text, json_text, tw_cursor,time,  favorite, reply, retweet) values\
                                    ( %(src_twid)s, %(twid)s, %(full_text)s, %(json_text)s, %(tw_cursor)s, %(time)s,%(favorite)s, %(reply)s, %(retweet)s)"
                            effect_row = db_cursor.execute(sql, {'src_twid': tweet_id\
                                                                , 'twid': timeline_add_entries_legacy.get("id_str", "")\
                                                                , 'full_text': timeline_add_entries_legacy.get("full_text", "")\
                                                                , 'json_text':str(timeline_add_entries_legacy)\
                                                                , 'tw_cursor':tw_cursor\
                                                                , 'time': timeline_add_entries_legacy.get("created_at", "")\
                                                                , 'favorite':timeline_add_entries_legacy.get("favorite_count", 0)\
                                                                , 'reply':timeline_add_entries_legacy.get("reply_count", 0)\
                                                                , 'retweet':timeline_add_entries_legacy.get("retweet_count", 0)})
                            if effect_row <=0:
                                print('数据库操作错误')

    return cursor_value, status_code

# * ------------- 程序开始 ---------------

# 要采集的推特用户id
user_id = ''

# 连接mysql数据库
try:
    conn = pymysql.connect(host="127.0.0.1", port=3306, user='root', passwd='root', db='tw', charset='utf8mb4',cursorclass=pymysql.cursors.DictCursor)
    conn.autocommit(True)
    db_cursor = conn.cursor()
    print("数据库连接成功！")
except Exception as e:
    print("数据库连接失败，错误信息：", e)

# 循环读取数据库中的twid
offset = 0
batch_size = 10
while True:
    # 使用 LIMIT 分页查询，按 offset 和 batch_size 读取数据
    sql = "SELECT twid FROM tweets LIMIT %s OFFSET %s"
    db_cursor.execute(sql, (batch_size, offset))
    result = db_cursor.fetchall()
    
    if not result:
        print('采集完成，程序退出')
        break
    
    for row in result:
        # 拉取该贴文的所有回复
        tw_cursor = ''
        n = 1
        while True:
            print('正在采集 '+ row['twid']+' 贴文，第'+ str(n) + ' 页，cursor:'+tw_cursor)
            tw_cursor_t, status_code = x_request(row['twid'], user_id, tw_cursor, db_cursor)
            if tw_cursor_t == "" and status_code != 200:
                print("网络异常，代码"+str(status_code)+" ，将重新请求...")
                sleep_time = random.uniform(5, 6)
                time.sleep(sleep_time)
                continue
            elif tw_cursor_t == "" and status_code == 200:
                print('贴文'+row['twid']+'采集结束')
                break
                
            tw_cursor = tw_cursor_t
            n += 1
            # 随机等待 1 到 3 秒
            sleep_time = random.uniform(3, 5)
            time.sleep(sleep_time)
    
    # 更新偏移量
    offset += batch_size

# 关闭mysql连接
conn.commit()
db_cursor.close()
conn.close()
